#!/usr/bin/env python
# -*- coding: UTF-8 -*-


import csv
import NMutl
import NMre
import datetime
import NMhtml

def foo(n, what, skipValidations=True, rowkey=None, colkey=None):
    if 2 and what == 2:
        b=NMtable() 
        b.CSVload('/home/milon/WFBscrap_CSVdump.csv',1, skipValidations)
    for dummy in range(0, n):
        if what == 1: b=NMtable()
        b.CSVload('/home/milon/WFBscrap_CSVdump.csv',what, skipValidations,rowkey=None, colkey=None )
    return b

def excptn(obj,val):
    raise Exception ('Err: '+str(type(obj)),val)
 
def ValidationException(obj, descr='', key='', val='', tp=''):
    print 'Err Validation from', type(obj) ,  descr ,  'key:', key ,'val:', val, 'valid types:',  tp
    raise Exception ('Err Validation from: ', type(obj) ,  descr ,  'key:', key ,'val:', val, 'valid types:',  tp)
    return obj

class NMpsdict(list):
    """a minimal pseudo dictionary based on list of lists where lists[0] is a key  - helps coz dict is not orderedl"""
    __slots__ = []
    def __init__(self,*args, **kwargs):
        list.__init__(self,*args, **kwargs)
    def __getitem__(self,key):
        rt=self.getkv(key)
        if rt is None:raise Exception ,  'KeyError:/t'+str(key)
        else:         return rt[1]  
    def get(self,key,ifNotfound):
        rt=self.getkv(key)
        if rt is None:return ifNotfound
        else:         return rt[1]            
    def getkv(self,key):
        rt= [item for item in self if item[0]==key]
        if len(rt) !=0 :return rt[0]
        else:           return None
    def keys(self):
        return [item[0] for item in self]
    def values(self):
        return [item[1] for item in self]

class WBC(str): 
    separator='.'
    colname=''
    def __init__(self,val): 
        str.__init__(self,val)
        #self.separator=separator
        #self.colname = colname
        self.level = len(self.split(self.separator))-1
        self.parent = '.'.join(val.split(self.separator)[0:-1])
    def getLevel (self,level):return '' if level > self.level else self.split(self.separator)[level] 
    def getRight(self, level):return '' if level > self.level else self.separator.join(self.split(self.separator)[level+1:])
    def getLeft(self, level):return '' if level > self.level else self.separator.join(self.split(self.separator)[:level])     
    def isChild(self, wbc):        return self.isDescentant(wbc) and wbc.level == self.level +1     
    def isDescentant(self, wbc):   return wbc.startswith(self + self.separator)
    def isSibling(self, wbc):      return self.parent == wbc.parent
    def siblings(self, wbcLst):    return [item for item in wbcLst if self.isSibling(item)]
    def children (self, wbcLst):   return [item for item in wbcLst if self.isChild(item)] 
    def descentants (self, wbcLst):return [item for item in wbcLst if self.isDescentant(item)]
    def root(self): return self.split(self.separator)[0]
    #      def __cmp__(self, other): 
    #        if str(self) < str(other):
    #            return -1
    #        elif str(self) > str(other):
    #            return 1
    #        else:return 0
     
class NMdict(dict):
    __slots__ = []
    ClsSkipValidation = False                 
    def __init__(self,dictarg={}):            #if values initialized on creation bypass avalidation ?
        dict.__init__(self,dictarg)
    def __setitem__(self,key,val):
        if GLNMdictSkipValidation or self.ClsSkipValidation or self.validates(key,val):dict.__setitem__(self,key,val)
        else:excptn(self,('Not Valid',key,val))
    def __repr__(self):return NMutl.tp_lastpart(type(self)) + '(' + dict.__repr__(self) + ')'
    def validates(self,key,val):
        return True 
    def vk(self):return [[self[k], k] for k in self.keys()]
    def sortValsByKey(self):return [self[k] for k in sorted (self)]
class NMlng(NMdict):
    """ a multi language object based on dict"""
    __slots__ = ('DefaultLang') 
    def __init__(self,val,lang='en' , DefaultLang='en'):
        if isinstance(val,  dict):
            NMdict.__init__(self, val)
        else:                   
            NMdict.__init__(self)
            self.__setitem__(lang,val)
        #super(type(self), self).__init__(self)                 
        self.DefaultLang=DefaultLang
         
    def __setitem__(self,lang,val):                                        #ToDo lang must raise error if not form a list of possible locals
        if lang==None:lang=self.DefaultLang
        NMdict.__setitem__(self, lang, val)
    def setitems(self,itemdict):
        for k in itemdict.keys():self[k]=itemdict[k]
    def getOrDefaultVal(self,lang): return self.get(lang,self.getDef)
    def getDefaultVal(self):        return self.get(self.DefaultLang,None)
    def setDefaultVal(self, val):self[self.DefaultLang]=val
    def validates(self,key,val):    return len(key)==2
    @classmethod
    def NMlngOrVal(cls,val,lang):return (val[lang] if isinstance(val,NMlng) else val)
class NMlngS(NMlng):
    """ a multi language string based on dict"""
    __slots__ = [] 
    def __init__(self,strval=u'',lang='en' , DefaultLang='en'):
        NMlng.__init__(self, strval, lang)
    def validates(self,key,val):
        return NMlng.validates(self, key, val) and isinstance(val,basestring)         


class NMcolumn(NMdict):
    __slots__ =  []
    def __init__(self, Name=None, Tp=object, Title='',Description='',Unit='',Ordinal=-1,shortName='',Fields=[]):
        NMdict.__init__(self) 
        #UsrObj=0
        _info_={}
        for l in NMcolumnValidkeys:
            if   l[0] == 'Tp':  self[l[0]]=eval(l[0]) 
            elif l[0] == 'Name':self.SetName(Name)
            else:      
                if isinstance(eval(l[0]), l[1]):   self[l[0]]=eval(l[0])
                else:                              self[l[0]]=l[1](eval(l[0]))   #convert it to type
    @classmethod
    def validates(cls,key,val):
        if key == 'Tp' and isinstance(val, (tuple, list)):
            return all([isinstance (it,type) for it in NMutl.ut_itemise(val,[], incltypes=(object))])
        if key == 'Name' and (val == None or isinstance(val,  (basestring, int))):return True                 #None allowed temporary 
        else:
            tst=NMcolumnValidkeys.getkv(key)
            return (not(tst is None))  and  isinstance(val,tst[1]) 
    def info(self):    return self[NMtblInfoName]
    def SetName       (self, val):    self['Name']=val
    def GetName       (self):return   self['Name']
    def GetTitle      (self):return   self['Title']
    def GetOrdinal    (self):return   self['Ordinal'] 
    def getShortName  (self):return   self['shortName']
    def GetUnit       (self):return   self['Unit']
    def SetOrdinal    (self, val):    self['Ordinal']=val
    def GetTp         (self):return   self['Tp']
    
class NMcolumns(NMdict):
    def validates(self,key,val):
        return isinstance(key, (basestring, int)) and isinstance(val,NMcolumn) and not(val.GetName() in self)
class NMRow(NMdict):
    __slots__ =  []
    def __init__(self, Key=None, RowNum=None,TMc=None,  dictarg={}):
        NMdict.__init__(self,dictarg)
        if dictarg == {} or  NMtblInfoName not in dictarg:
            if TMc is None:TMc=datetime.datetime.now()
            self[NMtblInfoName]=NMdict({'K':Key,'RN':RowNum, 'TMc':TMc})
    def info(self):    return self[NMtblInfoName]
    def infoKey(self): return self[NMtblInfoName]['K']
    def infoRN(self):  return self[NMtblInfoName]['RN']
    def infoTmCr(self):return self[NMtblInfoName]['TMc']
    def validates(self,key,val):
        return True
      
class NMtable(NMdict):
    """keys for rows & columns can not be None"""
    def __init__(self,TableName='mytbl',dictarg={}):
        NMdict.__init__(self,dictarg)
        self.Columns=NMcolumns()
        self.TableName=TableName
        self.AddColumn(WBC(NMtblInfoName),Ordinal=-999,Description=u'this keeps row housekeeping info',Tp=NMdict, shortName='-999')
        self.missingvalue='@N/A@'
    def defaultpath(self,dir='',  addstr='', extn=''):
        if dir=='':dir='/tmp/'
        return dir + self.TableName + addstr +'.' + extn
    def hasRowColumn(self, RowKey, ColumnKey):return RowKey in self and self.hasColumn(ColumnKey)
    def hasColumn(self, ColumnKey):return ColumnKey in self.Columns 
    def validates(self,key,val):
        return isinstance(val,NMRow) and not(key in self)
    def AddColumn(self,*args, **kargs):
        col=NMcolumn(*args, **kargs)
        if col.GetName() == None: col.SetName(len(self.Columns)-1)
        if col.GetOrdinal() == -1:col.SetOrdinal(len(self.Columns)-1)
        self.Columns[col.GetName()]=col
    def AddRow(self,key=None, RowNum=None, DTc=datetime.datetime.now(), contents={}):
        if RowNum == None:RowNum = len(self)+1                                     
        if key == None:key = RowNum
        self[key] = NMRow(key,RowNum, DTc, contents)
    def SetValAuto(self,key,colname,val):
        if not(key in self):                 self.AddRow(key)
        if not(colname in self.Columns):     self.AddColumn(colname)
        self[key][colname]=val
    def isValidVal(self, RowKey, ColumnKey, val):
        def lstvalid(vall, tpl):
            return (len(tpl) == len(vall) or len(val) == 0) and all([isinstance(it[0],it[1]) or it[0] == None for it in zip(vall, tpl)])
        if self.hasRowColumn(RowKey, ColumnKey):
            validTp=self.Columns[ColumnKey].GetTp()
            if isinstance(validTp,(list, tuple)) and isinstance(val,(list,  tuple)):
                if len(validTp) == 1 and len(val) > 1:validTp=validTp * len(val)
                if isinstance(validTp[0], list):
                    return len(validTp) == len(val) and all([lstvalid(it[0], it[1]) for it in zip(val, validTp)])
                else:return lstvalid(val, validTp)
            elif isinstance(val, validTp):return True
            else:return False
        else:return False
    def SetVal(self,RowKey, ColumnKey, val):
        if self.isValidVal(RowKey, ColumnKey, val):self[RowKey][ColumnKey]=val
        else:ValidationException(self,'Ilegal key or value type:' +  str(RowKey) +'|'+ str(ColumnKey),val=val, tp=self.Columns[ColumnKey].GetTp())
    def SetValFld(self,RowKey, ColumnKey,subfield, val):
        if self.hasRowColumn(RowKey, ColumnKey):
            if isinstance(self.Columns[ColumnKey].GetTp(),list):
                orgval=list(self[RowKey][ColumnKey])           #just a copy
                orgval[subfield]=val
                self.SetVal(RowKey, ColumnKey, orgval) 
    def getColKeyByShortName(self, shortName):
        for col in self.Columns.keys():
            if self.Columns[col].getShortName() == shortName:
                return col
        return None
    def diff(self, other):
        #!if None could be  a real value then it will no work 
        return [[k,c, self[k].get(c,None), other[k].get(c,None)] for c in self.Columns for k in self.keys() if self[k].get(c,None)!=other[k].get(c,None)]
    def same(self, other):
        return all(self[k].get(c,None)==other[k].get(c,None) for c in self.Columns for k in self.keys())  
    def GetRowListVK (self,fld,substNA=None):           return[[self[k].get(fld,substNA), k] for k in  self.keys()]
    def GetRowListSortedVK (self,fld,substNA=None):     return sorted(self.GetRowsList (fld,substNA=None))
    def GetRowKeysSorted (self,fld):                    return [l[0] for l in self.GetRowsListSortedVK(fld)]
    def GetRowKeysSortedByRInfo (self, rinfoFld='RN'):  return [l[1] for l in sorted([[self[k]['_Rinfo_'][rinfoFld],k]  for k in  self.keys()])]
    def ColumnKeysByOrdinal(self,colkeys=[]):           return self.ColumnKeysByFld('Ordinal',colkeys)
    def ColumnKeysByWBC(self,colkeys=[]):   
        return self.ColumnKeysSorted(colkeys)           # temporary for backwards compatibility   to be rplaced by  ColumnKeysSorted    
    def ColumnKeysSorted(self, colkeys):
        if colkeys==[]:colkeys = self.Columns.keys() 
        colkeys.sort()
        return colkeys    
    def ColumnKeysByFld(self,fldname,colkeys=[]):
        if colkeys ==[]:colkeys = self.Columns.keys()   
        return [dummy[1] for dummy in sorted([[self.Columns[dummy][fldname],dummy] for dummy in colkeys])]
    def filter(self, afunction):
        return [k for k in self.keys() if afunction(self[k])]
    def filterOnColumn(self, col, afunction):
        #r=n.filterOnColumn(x,'Currency@code',lambda y:y['en']==u'EUR')
        return [k for k in self.keys() if col in self[k] and afunction(self[k][col])]
    def filterColumns(self, afunction):
        return [k for k in self.Columns.keys() if afunction(self.Columns[k])]
    def filterAll(self, fk=None, fc=None, fv=None):
        return [l for l in self.genKCVall() if (fk is None or fk(l[0])) and (fc is None or fc(l[1]))  and (fv is None or fv(l[2]))]
    def filterAllVals(self, fv):
        return [l for l in self.genKCVall() if fv(l[2])]
    def filterMissing(self):
        return [l for l in self.genKCVall() if l[2]==self.missingvalue]
    def filterSet(self,setf,  fk=None, fc=None, fv=None): 
        for l in  self.genKCV():
            if (fk is None or fk(l[0])) and (fc is None or fc(l[1]))  and (fv is None or fv(l[2])):
                self[l[0]][l[1]]=setf(self[l[0]][l[1]])
    def filterDel(self, fk=None, fc=None, fv=None, delKey=False): 
        for l in  self.genKCV():
            if (fk is None or fk(l[0])) and (fc is None or fc(l[1]))  and (fv is None or fv(l[2])):
                if delKey:del self[l[0]]
                else:     del self[l[0]][l[1]]            
    def genColumns(self, columns=None):
        if columns is None: columns=self.ColumnKeysByOrdinal()
        for column in columns:
            for key in NMcolumnValidkeys:
                yield [column, key[0], self.Columns[column][key[0]] ]
    def genKCV(self, keys=None, columns=None):
        if keys is None:keys=self.keys()
        if columns is None: columns=self.Columns
        for key in keys:
            for column in columns:
                if column in self[key]:yield [key, column, self[key][column]]
    def genKCVall(self,keys=None, columns=None):
        if keys is None:keys=self.keys()
        if columns is None: columns=self.Columns
        for key in keys:
            for column in columns:
                yield [key, column, self[key].get(column,self.missingvalue)]
    def genCKVall(self,keys=None, columns=None):
        if keys is None:keys=self.keys()
        if columns is None: columns=self.Columns
        for column in columns:
            for key in keys:
                yield [column, key,  self[key].get(column,self.missingvalue)]
    def genCKV(self, keys=None, columns=None):
        if keys is None:keys=self.keys()
        if columns is None: columns=self.ColumnKeysByOrdinal()
        for column in columns:
            for key in keys:
                if  column in self[key]:yield [column, key,  self[key][column]]
 
    def getColTreeFun(self, col, funIf, funWhat):  
        """ example: getColTreeFun(l[0],lambda x: (not x.endswith('ddfff' )), lambda y: x.Columns[y]['shortName']) """ 
        if funIf(col): 
            return [funWhat(col), [self.getColTreeFun(clm , funIf, funWhat) for clm in self.getColChildren(col) if funIf(clm)]  ]
    def getColTreeFunOld(self, col, funIf, funWhat):  
        """ example: getColTreeFun(l[0],lambda x: (not x.endswith('ddfff' )), lambda y: x.Columns[y]['shortName']) """
        if funIf(col):
            ch = self.getColChildren(col)
            if len(ch) == 0 :  return funWhat(col)
            else :             return [funWhat(col), [self.getColTreeFun(clm , funIf, funWhat) for clm in ch if funIf(clm)]  ]
         
    def getColTree(self, col):  
        ch = self.getColChildren(col)
        if len(ch) == 0 :  return col
        else :             return [col, [self.getColTree(clm) for clm in ch]  ]       

    def genColTree(self, col, funIf, funWhat):   
        def valOrGen(vl):
            if str(type(vl)).find("generator")>-1: #if isinstance(vl, generator) does not work 
                for dummy in vl: valOrGen(vl)
            else:
                return vl
        if funIf(col):
            ch = self.getColChildren(col)
            if len(ch) ==0 :  yield funWhat(col)
            else :            
                rt = [funWhat(col), [self.genColTree(clm , funIf, funWhat) for clm in ch if funIf(clm)]  ]   
                for it in rt: yield it 
               
    def genColTreeLst(self, col, funIf, funWhat): 
        lst=[]
        for vl in self.genColTree(col, funIf, funWhat):lst.append(vl) 
        return lst 

    
    def getColParent(self, col):    return self.Columns.get(col.parent, None) 
    def getColChildren(self, col):  return sorted ([clm for clm in self.Columns if clm.parent == col]) 
    def getColSiblings(self, col):  return sorted ([clm for clm in self.Columns if clm.parent == col.parent])
    def getColDescedants(self, col):return sorted(col.descentants(self.Columns))
    
    def CSVdumpRC(self, pathname=''):
        if pathname=='':pathname=self.defaultpath('','_CSVdumpRC','csv')
        fl = open(pathname,'w')
        CSVheaders=['RT', 'Key', 'Val'] 
        dout = csv.DictWriter(fl, CSVheaders, restval='', extrasaction='ignore')
        #dout.writerow(dict(zip(CSVheaders, CSVheaders))
        for key in self.Columns:
            if self.Columns[key].GetOrdinal() > -1:
                dout.writerow({'RT':1, 'Key':key, 'Val':self.Columns[key]})
        for key in self.keys():
            dout.writerow({'RT':2, 'Key':key, 'Val':self[key]}) 
        fl.close()  
        return pathname       
    def CSVloadRC(self, pathname='', skipValidations=True):
        global GLNMdictSkipValidation
        recompiletype = NMre.re.compile("(<class.'.*?|<type.'|'>)",  NMre.re.VERBOSE)
        if skipValidations:GLNMdictSkipValidation = True 
        if pathname=='':pathname=self.defaultpath('','_CSVdumpRC','csv')
        fl = open(pathname,'r')
        dr = csv.reader(fl)
        for line in dr:
            [RT, Key, Val] = [line[0], line[1], line[2]]
            if RT=='2':
                Val=eval(Val)
                self[Key] = NMRow(dictarg=Val)
            else:
                #print Val, '============',  recompiletype.sub('',Val,  0)
                Val= recompiletype.sub('',Val,  0).replace('NMtbl.', '') 
                Val=eval(Val)
                self.Columns[Key]=Val
        fl.close()
        if skipValidations:GLNMdictSkipValidation = False
                     
    def CSVdump(self, pathname='',  DoNotRepeatCols=True, InclHidenCols=False, dumpwhat=3):
        """InclHidenCols default Columns not really neededed since created automaticly on table creation
           DoNotRepeatCols = True to save space
           dumpwhat 1 = structure 2 = data, 3 = both
        """
        recompiletype = NMre.re.compile("<class.*?\.|<.*?\s|'|>",  NMre.re.VERBOSE)
        def dump(gen, rt):
            ColLast = ''
            dic={'RT':rt}
            for item  in gen:
                if self.Columns[item[0]].GetOrdinal() > -1 or InclHidenCols:
                    if rt == 1 and item[1]=='Tp':
                        item[2]=recompiletype.sub('',str(item[2]),  0)
                    NMutl.dic_setFromLsts(dic, CSVheaders[1:], ['' if (ColLast == item[0] and DoNotRepeatCols) else item[0],item[1], item[2]])
                    dout.writerow(dic)
                ColLast=item[0]          
        if pathname=='':pathname=self.defaultpath('','_CSVdump','csv')
        fileOut = open(pathname,'w')
        #ColKeys = self.ColumnKeysByOrdinal()
        CSVheaders=['RT','Column', 'Key', 'Val'] 
        dout = csv.DictWriter(fileOut, CSVheaders, restval='', extrasaction='ignore') 
        dout.writerow(dict(zip(CSVheaders, CSVheaders)))
        if 1 & dumpwhat == 1:dump(self.genColumns(), 1)
        if 2 & dumpwhat == 2:dump(self.genCKV(), 2)
        fileOut.close()
        return pathname    
    
    def CSVload(self, pathname='', loadwhat=3, skipValidations=False,  rowkey=None, colkey=None):
        def loadsetcolumn(): 
            if Key == 'Name' or Key == 'shortName': vl=Val
            else:vl=eval(Val)
            self.Columns[Column][Key]=vl 
        global GLNMdictSkipValidation
        Mtblldflgk='_ldflg_'
        if rowkey in self and self[rowkey].info().get(Mtblldflgk, False):return True
        if colkey in self.Columns and self.Columns[colkey].info().get(Mtblldflgk, False):return True
        if pathname == '':pathname= self.defaultpath('', '_CSVdump','.csv')
        fin=open(pathname)
        dr = csv.reader  (fin)
        #headersLst=dr.next()
        dr.next()
        ColumnFound=False
        
        if skipValidations:GLNMdictSkipValidation = True 
        for line in dr:
            [RT, Key, Val] = [line[0], line[2], line[3]]
            if line[1] != '':
                Column = line[1]
                if RT == '2':
                    Tp=self.Columns[Column]['Tp']
                    if ColumnFound and ColumnFound != Column:break
                    if colkey==Column:ColumnFound=Column
            if RT == '2':
                if (2 & loadwhat == 2): 
                    if (not rowkey is None and  rowkey != Key) or (not colkey is None and colkey != Column):continue
                    if not Key in self:self.AddRow(Key)
                    if not Column in self[Key]:
                        if Tp == unicode or Tp == int or Tp == float:self[Key][Column]=Tp(Val)
                        elif Tp == datetime.date:  
                            Val= map(int, Val.split('-'))
                            self[Key][Column]=datetime.date (Val[0], Val[1], Val[2])  
                        elif Tp == str:self[Key][Column]=Val
                        else:
                            self[Key][Column] = eval(Val)
                else: break 
            elif (1 & loadwhat) == 1:
                if not Column in self.Columns:self.AddColumn(WBC(Column))   
                loadsetcolumn ()
        fin.close()
        if rowkey in self:
            self[rowkey].info()[Mtblldflgk]=True
            return rowkey
        elif colkey in self.Columns: 
            self.Columns[colkey].info()[Mtblldflgk]=True 
            return colkey 
        else :
            for col in self.Columns: self.Columns[col].info()[Mtblldflgk]=True 
            for row in self:self[row].info()[Mtblldflgk]=True 
        if skipValidations:GLNMdictSkipValidation = False   
    def GetLangOfColDefinitions(self):
        """returns a set of all Languages used in Column Definitions """
        lang=[]
        for col in self.Columns.keys() :
            [lang.extend(self.Columns[col][k].keys()) for k in self.Columns[col].keys() if isinstance(self.Columns[col][k], NMlng)]
        return list(set(lang))
    def RowKeysByFld(self,fldname):return [dummy[1] for dummy in sorted([[self[dummy][fldname],dummy] for dummy in  self.keys()])] 
    def ValtoStr(self,key,col):
        if key in self:
            vl=self[key].get(col, self.missingvalue)
        else:vl=self.missingvalue  
        return self._ValtoStr(vl)
    def _ValtoStr(self, vl):
        if isinstance(vl, float):
            return round(vl,2)
        elif isinstance(vl, list): 
            return '\n'.join([self._ValtoStr(l) for l in vl])
        else: 
            return vl  
    def GetHeadersDic(self,colkeys=[],lang='en'):
        if colkeys == []:colkeys=self.ColumnKeysByOrdinal()
        Headers=NMdict()
        for kv in NMcolumnValidkeys:
            vl=[NMlng.NMlngOrVal(self.Columns[ck][kv[0]],lang)  for ck in colkeys]
            Headers['__'+kv[0]+'__']=NMdict(zip(colkeys,vl)) 
        return Headers
    #def toCSVbeforeprint(self, type, val):
        pass
    def toCSV(self,file='*',colkeys=[],rowkeys=[],lang='en', headersLst=['__Name__'],type=2,sort2=False):
        """type can be '' (returns only headers) 2 4 8  result  # if  file==False no print occurs"""
        if file:
            if file=='*':file= self.defaultpath('', str(type),'csv')
            fileOut = open(file, 'w') 
        if colkeys == []:colkeys=self.ColumnKeysByOrdinal()  
        if rowkeys == []:rowkeys =sorted(self.keys())
        rt=None
        if headersLst:
            rt=self.GetHeadersDic(colkeys,lang)
            rt=[rt[k] for k in headersLst]
        else:
            rt=[]
        if type == 2 and file:                                         #Normal table structure 
            dout = csv.DictWriter(fileOut, colkeys, restval=self.missingvalue, extrasaction='ignore') 
            dout.writerows(rt)
            ks=self.keys()
            if sort2:ks=sorted(ks)
            for k in ks:dout.writerow(self[k])
        if type ==4:                                          #Vertical single column table 
            cols=('Key','Fld','Val')
            rt=[dict(zip(cols,['__'+ k +'__' for k in cols]))]
            gen=self.genKCV(rowkeys, colkeys)
            for it in gen:
                rt.append(NMutl.dic_setFromLsts({}, cols, [it[0], it[1],it[2]]))
            if file:
                dout = csv.DictWriter(fileOut, cols , restval=self.missingvalue, extrasaction='ignore')    
                dout.writerows(rt)
        if type == 8:                                                         #unic values
            rt={}
            rawstrComments=     r"""(@\D+@)"""
            for k in rowkeys:
                for c in self[k].keys():
                    if c != NMtblInfoName and c.split('@')[-1]!='date':
                        if rt.get(c,None)== None:rt[c]={}
                        items=NMutl.ut_itemise(self[k][c],[])
                        tmp=[]
                        for it in items:
                            Comment=NMre.reFindReplace(rawstrComments,'',it)
                            if Comment[2] !=0: tmp.append(Comment[0][0])  
                            else:              tmp.append(Comment[1])  
                        for it in tmp:
                            if rt[c].get(it):rt[c][it]=rt[c][it]+1 
                            else:            rt[c][it]=1               
            if file:          
                dout= csv.writer (fileOut)
                dout.writerow(('_FldName','_words','_cnt'))
            for k1 in rt.keys():
                ks=sorted(rt[k1].keys())
                for k2 in ks:
                    if file:dout.writerow((k1,k2,rt[k1][k2]))    
        else:pass
        if file:fileOut.close()
        return rt
class NMtableGeo(NMtable):
    _cnstNBS="&nbsp;" 
    """keys for rows & columns can not be None"""
    def __init__(self,TableName='mytbl',dictarg={}):
        NMtable.__init__(self,TableName, dictarg)  
        self.missingvalue=self._cnstNBS #! + '-' + self._cnstNBS
        #self.AutoLinkCntrCols = [self.getColKeyByShortName(x) for x in  ['LandBound', 'ExportPartners', 'ImportPartners']] 
        self.AutoLinkCntrCols = ['&nbsp;.02.05', '&nbsp;.05.11.02', '&nbsp;.05.12.02']
    def getCountryCode(self, countryName): 
        for cntr in self: 
            if self[cntr]['&nbsp;']['ccs'][-1]== countryName:
                return cntr
        else: return None
        
    def countryRect(self, country, cropPerc=1):
        rect=  self[country].get('&nbsp;.02.02.02', None)  
        if rect is None:return[]
        else:
            if cropPerc == 1:return rect
            else:
                widthC=(abs(rect[2]-rect[0]) * cropPerc)/2
                heightC=(abs(rect[1]-rect[3]) * cropPerc)/2 
                return  [rect[0] + widthC,rect[1] - heightC, rect[2] - widthC , rect[3] + heightC ]
    def EnBorderCountries (self, CC):
        BC=self[CC].get('&nbsp;.02.05', False)
        if BC:
            BC=self._strGet(BC,'en')
            BC=[str(l[0]) for l in BC] 
            if len(BC)>1 :return  BC[1:]
            else:return []
        else:return []    
    def EnRegionsAndCities(self, CC):
        RC=self._strGet(self[CC].get('&nbsp;.04.05', ''),'en').split(',')  
        return [str(item).strip() for item in RC]
    def EnCapital(self, CC):
        cap= self[CC].get('&nbsp;.04.06', False)
        if cap and cap[0]:return cap[0]['en']
        else: return ''   
        
    def countryAproxCoord(self, country):
        center = self[country].get('&nbsp;.02.02.01', None) 
        if center is None: 
            center = self[country].get('&nbsp;.04.06', None) 
            if center is not None:center = center[1]   
        if center is None:return None
        else: 
            offset = self[country].get('&nbsp;.02.04', None)
            if offset is None:offset= 1
            else:
                offset = (offset[0] / 3.14) /100000  #  0.000015
                if offset >  10 : offset=10
            center=center.split(' ') 
            center = "%s.%s,%s.%s" %( center[0],center[1], center[3], center[4] ) 
            return(center, "%2.2f" % offset)
    def toHtml(self, lng='en', cols=None): 
        if cols is None:cols=[item for item in self.ColumnKeysByWBC() if not item.endswith('@date')]
        headlst = [self.Columns[item].GetTitle()[lng] for item in cols] 
        rowlst  = [[self.getHtml(row, col, lng) for col in cols] for row in self] 
        htmltable=NMhtml.table(contentslst = rowlst, attributes='border=\"4px\"',caption = lng, headerlst = headlst, footerlst= False, tagColums=False) 
        htmltable.setAttr('cellpadding', '2px')
        return  htmltable
        
    def getHtmlTbl(self, key, colNames, lng):
        res=[]
        for col in colNames:
            rt = self[key].get(col,False)
            if rt: 
                val= self._strGet(rt,lng)
                if  isinstance(val, list): 
                    Title=self.Columns[col]['Title'][lng] 
                    cnt=-1
                    for el in val: 
                        cnt += 1
                        if el != self.missingvalue:
                            ttl= Title + " - "+ str(self.Columns[col]['Fields'][lng][cnt]  ) 
                            res.append([ttl, el, self.Columns[col]['Unit'][lng]])   
                else:
                    res.append([self.Columns[col]['Title'][lng], val, self.Columns[col]['Unit'][lng]])    
        return NMhtml.table(res) #, caption = res[0][0])
    
    def getGVTable(self, key, col, lng): 
        if key in self and col in self[key]:
            vl=self[key][col] 
            if isinstance(vl, (NMlngS, NMlng)): vl=vl[lng] 
            elif isinstance(vl, list) and len(vl)> 0:
                if isinstance (vl[0], list):
                    vl=[self._valGet(it,  lng) for it in vl] 
                    vl=[[[it] for it in item] for item in vl] ##  if item[1]!= self.missingvalue] 
                    return   NMhtml.GVtable(self.Columns[col].getShortName(),  vl)  
            if len(self.Columns[col]['Fields'][lng]) > 0:  
                vl=[self.Columns[col]['Fields'][lng], vl] 
                vl =[[[it] for it in item] for item in NMhtml.lst_transpose(vl)] 
                return  NMhtml.GVtable(self.Columns[col].getShortName(),  vl)  
        else:
            return None    
    def _valGet(self, vl,lng ): 
        if isinstance(vl, (NMlngS, NMlng)):  return self._valGet(vl[lng],lng)  
        elif vl is None:                     return self.missingvalue  
        elif isinstance(vl, list):           return [self._valGet(l, lng) for l in vl]
        else:return vl
    def getCntrCCSorted(self, lng):
        rt=[[self[k]['&nbsp;']['country'][lng], k] for k  in self.keys()  ] 
        rt.sort()
        return rt 
    
#    def getHtmlKeyOption(self,cntr, lng, select=False, cnt=False):   
#         vl=self[cntr]['&nbsp;']
#         vl=NMhtml.option(vl['country'][lng] + ' ('+ vl['ccs'][0]+')' ,'', vl['ccs'][0], select)
#         if cnt:vl.attrAppendCnt(cnt)
#         return vl
    def getHtmlKeyOptions(self,lng, selected=False):
        cntrCC=self.getCntrCCSorted(lng)
        #ol= [NMhtml.option( v[0]+ ' ('+ v[1]+')', '', v[0], v[1] == selected)  for v in cntrCC ]
        ol= [NMhtml.option( "%s%s%s%s" %(v[0] , ' (',  v[1], ')'), '', v[1], v[1] == selected)  for v in cntrCC ]
        cnt=0
        for v in ol :
            cnt+=1
            v.attrAppendCnt(cnt)
        ##contents='', attributes='',  value=False, selected=False
        return ol
#        kl = self.keys() 
#        #kl = [ [cntr  , k ]  for k in self.keys()]
#        kl = [self.getHtmlKeyOption(k, lng, k==selected) for k in kl]    
#        kl.sort((lambda x, y: cmp(x.contents.split(' ')[0] ,  y.contents.split(' ')[0]))) 
#        cnt=0
#        for k in kl :
#            cnt+=1
#            k.attrAppendCnt(cnt)
#        return ' '.join([str(r)[:-1] for r in kl])    
    
    def LatLngToMF(self, latlng): 
            rt=latlng.split(',')
            if rt[0].endswith('S'):rt[0]='-'+rt[0].strip()
            if rt[1].endswith('W'):rt[1]='-'+rt[1].strip()
            rt=[k[:-2].strip().replace(' ','.')  for k in rt] 
            return ';'.join(rt)
    
    
    def getHtml(self,key,col,lng, vl=None, referer="/"):   
        ##print ['getHtml', key, col, lng, vl, referer]
        def getAutoLink(): 
            if col in self.AutoLinkCntrCols:
                cnt=-1
                for l in vl:
                    cnt+=1
                    if len(l)==3 : 
                        # print [referer, l[2], l] 
                        l[0] =NMhtml.a("%s%s%s%s"  %(l[0],  ' (' ,  l[2],')'), 'class=\"nm-inlnk\"', referer [:-7] + l[2] + ".html" )  
                        vl[cnt]=l[:-1]
                        
        shortName=self.Columns[col].getShortName()
        Unit = self.Columns[col]['Unit'][lng] 
        def lstToTag(lst):
            rt = NMhtml.p()
            for it in lst:
                rt.insertContents(it)
            return rt     
            
        if key in self and col in self[key]: 
            if vl is None:vl = self._strGet(self[key][col],  lng)
            #if shortName=='GeoCoordCenter':vl = NMhtml.mformatGeoAbr(LatLngToMF(vl), vl)()  
            if isinstance(vl, list):##  and len(vl) >0:   how do we know len > 0  ?
                ##kati  trexei me to Nepal np edo print ['FFFFFFFFF', key, col, lng, vl ] 
                if isinstance (vl[0], list):
                    getAutoLink()  
                    #print [shortName,  vl]  
                    if Unit !='': [it.append(Unit) for it in vl if it[1] != self.missingvalue]
                #                if col == '&nbsp;.02.04':   #Area
                #                   vl=[it for it in vl if it[1] != self.missingvalue] 
                #print vl, self.missingvalue 
                #                if   listAsGVtable: 
                #                     vl=[[[it] for it in item] for item in vl]
                #                     return  NMhtml.GVtable(self.Columns[col].getShortName(),  vl)
                
                #                print [key, col, shortName]  
                #                if shortName=='LandBound':
                #                   print NMhtml.table(vl) , vl
                #                   print getHtmlAutoLink(vl)  
                    return NMhtml.table(vl)  
                else:  
                    #shortName=self.Columns[col].getShortName()
                    if self.Columns[col]['Tp'] == [str] and shortName != 'CountryName' or shortName=='NaturalRes':  
                        #divCont=NMhtml.div('', "class=\"nm-dv-cont\"")
                        #divCont.insertContents(NMhtml.div([NMhtml.span(it) for it in  vl],"class=\"nm-con-span\"")) 
                        vl.sort() 
                        return NMhtml.table(NMutl.lst_div(vl,2, ''), 'class="nm-tbl-2d"')
                        #return divCont
                        #return NMhtml.div(NMhtml.div(([NMhtml.span(it) for it in  vl], "class=\"nm-con-span\"")),) 
                        #return NMhtml.div ([NMhtml.span(it) for it in  vl], "class=\"nm-con-span\"")
                        #return NMhtml.ul ([NMhtml.p(it) for it in  vl])
                    elif len(self.Columns[col]['Fields'][lng]) > 0:  
                        if shortName=='Capital' and len(vl) > 2 and vl[1] !=self.missingvalue :  
                            vl[0]=NMhtml.mformatGeoAbr(self.LatLngToMF(vl[1]), vl[0])()  
                        rt = [[x[0], x[1]] for x in zip(self.Columns[col]['Fields'][lng],vl) if x[1] != self.missingvalue and  x[1] != ''] 
                        rt = self.getHtml(key, col, lng,rt, referer) 
                        return rt  
                    else: 
                        rt = NMhtml.div() 
                        map(lambda x: rt.insertContents(NMhtml.p(x)) , vl)  
                        return  rt  
            else:
                if shortName=='GeoCoordCenter':vl = NMhtml.mformatGeoAbr(self.LatLngToMF(vl), vl)()  
                rt= '%s%s%s' % (self._strGet(vl, lng), self._cnstNBS, self.Columns[col]['Unit'][lng] )
                return  rt.rstrip() 
        else:return self.missingvalue  
       
    def _strGet(self, vl,lng):
        def groupThousands(val, sep=self._cnstNBS): 
            if val < 0: return '- ' + groupThousands(-val, sep) 
            if val < 1000: return str(val) 
            return '%s%s%03d' % (groupThousands(val // 1000, sep), sep, val % 1000) 
        if isinstance(vl, (NMlngS, NMlng)):  return self._strGet(vl[lng],lng)
        elif isinstance(vl, float):          return "%2.2f" % vl  
        elif isinstance(vl, (int, long)):    return groupThousands(vl) #"%d" % vl
        elif vl is None:                     return self.missingvalue 
        elif isinstance(vl, datetime.date):  
            if vl.year==9999:vl=vl.replace(datetime.date.today().year) 
            return str(vl.year) if vl.day==31 and vl.month==12 else str(vl)##   .strftime('%B %d') #strftime works only for years after 1900
        elif isinstance(vl, list):           return [self._strGet(l, lng) for l in vl]
        else:return vl 
  
GLNMdictSkipValidation=False
NMtblInfoName='_info_'


NMcolumnValidkeys=NMpsdict([('Name',str),('Title',NMlngS),('Description',NMlngS),('Unit',NMlngS),\
                            ('Fields',NMlng),('shortName',str),('Ordinal',int),('Tp',type), ('UsrObj', object), (NMtblInfoName, dict)])

def tst(cnt):
    import NMmisc
    l=[]
    ll=[]
    for dummy in range(1, cnt):
        s=u'a'*100
        item= NMlng(s)
        item.setitems({'a1':s, 'a2':s,'a3':s, 'a4':s, 'a5':s,'a6':s,'a7':s, 'a8':s,'a9':s})
        ll.append(item)
        l.append(item.values())
        l.append(item.keys())
        NMmisc.pcl_tofile(l,'/home/milon/del_list')
        NMmisc.pcl_tofile(ll,'/home/milon/del_dir')
    return (l, ll)
class tsttable(NMtable):
    def __init__(self,TableName='tst'):
        #NMtable.__init__(self)
        super(tsttable, self).__init__(self)
